/**
 * Compute document frequency (number of documents in which each term
 * occurs) of terms. Optionally, restrict the computation to documents (pages)
 * in the given list; if no page_filter list is given, use all pages.
 *
 * Usage:
 * ... [page_filter] <adj
 *
 * page_filter has format:
 * wiki_article_ordinal
 * wiki_article_ordinal
 * ...
 *
 * adj has format:
 * wiki_article_ordinal num_terms term term ...
 * wiki_article_ordinal num_terms term term ...
 * ...
 * where each term is a non-negative integer.
 *
 * Output (on standard out):
 * term doc_freq
 * term doc_freq
 * ...
 */ 
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <set>
#include "u1.h"

using namespace std;

typedef map<int, int> doc_freq_t;

int main(int argc, char **argv)
{
  if (argc > 2) {
    TRACE<<"see comments for usage"<<endl;
    exit(EXIT_FAILURE);
  }
  set<int> page_filter;
  if (argc == 2) {
    string page_filter_file = argv[1];
    TRACE<<"Reading page filter from "<<page_filter_file<<endl;
    ifstream page_filter_in(page_filter_file.c_str());
    int article_num;
    while (page_filter_in>>article_num) {
      page_filter.insert(article_num);
    }
    page_filter_in.close();
    TRACE<<"Read "<<page_filter.size()<<" pages to filter."<<endl;
  } else {
    TRACE<<"Using all pages (no page filter)."<<endl;
  }

  TRACE<<"Reading adj from stdin and building terms list."<<endl;
  doc_freq_t doc_freq;
  int article_num, num_terms;
  while(cin>>article_num>>num_terms) {
    // Only process articles in the filter (if any filter was given).
    if (page_filter.empty() ||
        page_filter.find(article_num) != page_filter.end())
    {
      for (int i = 0; i < num_terms; ++i) {
        int term;
        if (!(cin>>term)) {
          TRACE<<"Expected "<<num_terms<<" terms for article "<<article_num
            <<" but got "<<i<<endl;
          exit(EXIT_FAILURE);
        }
        doc_freq_t::iterator it = doc_freq.find(term);
        if (it == doc_freq.end()) {
          doc_freq.insert(make_pair(term, 1));
        } else {
          it->second += 1;
        }
      }
    }
    else
    {
      // Just read in the terms.
      for (int i = 0; i < num_terms; ++i) {
        int term;
        if (!(cin>>term)) {
          TRACE<<"Expected "<<num_terms<<" terms for article "<<article_num
            <<" but got "<<i<<endl;
          exit(EXIT_FAILURE);
        }
      }
    }
  }

  TRACE<<"Writing "<<doc_freq.size()<<" terms to stdout."<<endl;
  for (doc_freq_t::const_iterator it = doc_freq.begin();
      it != doc_freq.end(); ++it) {
    cout<<it->first<<" "<<it->second<<'\n';
  }
  TRACE<<"Done."<<endl;
}
/*
* Copyright (c) 2009 John Lees-Miller
* 
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
* 
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* 
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/

